{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.text import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sentencepiece"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## train Hebrew Language Model on Wiki"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "after that, the TextList load all the text file from givven directory (so put attention to delete any unwanted text file inside this folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = open(\"data/wiki/wikipedia.txt\").read().split(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "spm = SPProcessor(lang=\"heb\", vocab_sz=5000)\n",
    "src = TextList.from_df(pd.DataFrame(p), processor=spm).random_split_by_pct(0.2).label_for_lm(ignore_empty=True, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the spm\n",
    "!cp tmp/spm.model ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "wiki_data = src.databunch(bs=64)\n",
    "# save the model and the data\n",
    "wiki_data.save('wiki_data_spm.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "functools.partial(<class 'fastai.text.models.awd_lstm.AWD_LSTM'>, n_layers=2)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "functools.partial(<class 'fastai.text.models.awd_lstm.AWD_LSTM'>, n_layers=2)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-1b6489ad66c9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# wiki_data = load_data(path, 'wiki_data_spm.pkl')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m# create the model from the data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mlearn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlanguage_model_learner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwiki_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpartial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mAWD_LSTM\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_layers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/text/learner.py\u001b[0m in \u001b[0;36mlanguage_model_learner\u001b[0;34m(data, arch, config, drop_mult, pretrained, pretrained_fnames, **learn_kwargs)\u001b[0m\n\u001b[1;32m    202\u001b[0m                            pretrained_fnames:OptStrTuple=None, **learn_kwargs) -> 'LanguageLearner':\n\u001b[1;32m    203\u001b[0m     \u001b[0;34m\"Create a `Learner` with a language model from `data` and `arch`.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 204\u001b[0;31m     \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_language_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0march\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitos\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdrop_mult\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdrop_mult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    205\u001b[0m     \u001b[0mmeta\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_model_meta\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0march\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    206\u001b[0m     \u001b[0mlearn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLanguageLearner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msplit_func\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmeta\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'split_lm'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mlearn_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/text/learner.py\u001b[0m in \u001b[0;36mget_language_model\u001b[0;34m(arch, vocab_sz, config, drop_mult)\u001b[0m\n\u001b[1;32m    187\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_language_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0march\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mCallable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvocab_sz\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdrop_mult\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1.\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    188\u001b[0m     \u001b[0;34m\"Create a language model from `arch` and its `config`, maybe `pretrained`.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 189\u001b[0;31m     \u001b[0mmeta\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_model_meta\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0march\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    190\u001b[0m     \u001b[0mconfig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mifnone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeta\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'config_lm'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    191\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: functools.partial(<class 'fastai.text.models.awd_lstm.AWD_LSTM'>, n_layers=2)"
     ]
    }
   ],
   "source": [
    "# wiki_data = load_data(path, 'wiki_data_spm.pkl')\n",
    "# create the model from the data\n",
    "learn = language_model_learner(wiki_data, AWD_LSTM)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>4.081431</td>\n",
       "      <td>3.855876</td>\n",
       "      <td>0.298150</td>\n",
       "      <td>32:36</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# we unfreeze all the model, beacuse he have totally random weight.\n",
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(1, 1e-2, moms=(0.8,0.7))\n",
    "learn.save('heb_spm_1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEGCAYAAACNaZVuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de3xU9Z3/8dcnkxuEQLhEBCJy0SoXIYR4q4pSrBXrXaqw8qi3SrW2buu6u7TdrbZdK922rG23P/3RKtrWQusF6/2OtVhEAgKCiKBcRBACyDVAbp/9Y05iEiYQwkzOZOb9fDx4ZHJy5pzPdya8853vOed7zN0REZH0khF2ASIi0vYU/iIiaUjhLyKShhT+IiJpSOEvIpKGMsMuoCV69Ojh/fr1C7sMEZF2ZcGCBVvcvTDWz9pF+Pfr14+ysrKwyxARaVfMbG1zP9Owj4hIGlL4i4ikIYW/iEgaahdj/iKSGqqqqli/fj379u0Lu5SUkpubS1FREVlZWS1+jsJfRNrM+vXryc/Pp1+/fphZ2OWkBHdn69atrF+/nv79+7f4eRr2EZE2s2/fPrp3767gjyMzo3v37of9aUrhLyJtSsEff615TVM6/F9Zvon/99qqsMsQEUk6KR3+r60o53d/Xx12GSKSJLZu3UpxcTHFxcUcffTR9OnTp/77ysrKFm3juuuuY8WKFQmuNPFS/oCvblYjInW6d+/OokWLALjzzjvp1KkTt99+e6N13B13JyMjdt94+vTpCa+zLaR0z19DiyLSEqtWrWLo0KHcdNNNlJSUsHHjRiZNmkRpaSlDhgzhRz/6Uf26Z555JosWLaK6upqCggImT57M8OHDOf3009m8eXOIrTg8qd/zD7sAEYnph08t490NO+O6zcG9O3PHRUNa9dx3332X6dOnc9999wEwZcoUunXrRnV1NaNHj2bcuHEMHjy40XN27NjB2WefzZQpU7jtttt44IEHmDx58hG3oy2kds8/7AJEpN0YOHAgJ598cv33M2bMoKSkhJKSEpYvX8677757wHM6dOjA2LFjARg5ciRr1qxpq3KPWOr3/NX1F0lKre2hJ0peXl7945UrV/LLX/6St956i4KCAiZOnBjzPPrs7Oz6x5FIhOrq6japNR5Su+evQX8RaYWdO3eSn59P586d2bhxIy+88ELYJcVdwsLfzB4ws81mtrTBsq+Y2TIzqzWz0kTtuyGd7SMih6ukpITBgwczdOhQbrzxRs4444ywS4o7S1Q4mtkoYDfwe3cfGiwbBNQC/x+43d1bdIeW0tJSb83NXO58chmPL1zPkju/dNjPFZH4W758OYMGDQq7jJQU67U1swXuHrOjnbAxf3d/3cz6NVm2PCgoUbsVEZEWSNoxfzObZGZlZlZWXl7e6u1o0EdE5EBJG/7uPs3dS929tLAw5v2HD0kfMEREYkva8I8bdf1FRA6Q0uFvusxLRCSmRJ7qOQOYC5xgZuvN7AYzu8zM1gOnA8+YWcJPnlXHX0TkQAkLf3ef4O693D3L3Yvc/X53nxU8znH3nu6e0HMwNeYvIg2dc845B1ywdc899/CNb3yj2ed06tQJgA0bNjBu3Lhmt3uo09HvueceKioq6r+/4IIL2L59e0tLj7uUHvYBXeQlIp+ZMGECM2fObLRs5syZTJgw4ZDP7d27N48++mir9900/J999lkKCgpavb0jldLhr46/iDQ0btw4nn76afbv3w/AmjVr2LBhA8XFxYwZM4aSkhJOOukk/vrXvx7w3DVr1jB06FAA9u7dy/jx4xk2bBhXXXUVe/furV/v5ptvrp8K+o477gDgV7/6FRs2bGD06NGMHj0agH79+rFlyxYApk6dytChQxk6dCj33HNP/f4GDRrEjTfeyJAhQzjvvPMa7edIpf7EbmEXICKxPTcZPnknvts8+iQYO6XZH3fv3p1TTjmF559/nksuuYSZM2dy1VVX0aFDB2bNmkXnzp3ZsmULp512GhdffHGzF6Tee++9dOzYkSVLlrBkyRJKSkrqf3bXXXfRrVs3ampqGDNmDEuWLOHWW29l6tSpzJ49mx49ejTa1oIFC5g+fTrz5s3D3Tn11FM5++yz6dq1KytXrmTGjBn89re/5corr+Sxxx5j4sSJcXmpUrvnr66/iDTRcOinbsjH3fne977HsGHDOPfcc/n444/ZtGlTs9t4/fXX60N42LBhDBs2rP5nf/nLXygpKWHEiBEsW7Ys5lTQDc2ZM4fLLruMvLw8OnXqxOWXX87f//53APr3709xcTEQ/ymjU7/nr66/SHI6SA89kS699FJuu+02Fi5cyN69eykpKeHBBx+kvLycBQsWkJWVRb9+/WJO4dxQrE8Fq1ev5uc//znz58+na9euXHvttYfczsGOS+bk5NQ/jkQicR32SfGev7r+ItJYp06dOOecc7j++uvrD/Tu2LGDo446iqysLGbPns3atWsPuo1Ro0bx8MMPA7B06VKWLFkCRKeCzsvLo0uXLmzatInnnnuu/jn5+fns2rUr5raeeOIJKioq2LNnD7NmzeKss86KV3Oblfo9f436i0gTEyZM4PLLL68f/rn66qu56KKLKC0tpbi4mBNPPPGgz7/55pu57rrrGDZsGMXFxZxyyikADB8+nBEjRjBkyBAGDBjQaCroSZMmMXbsWHr16sXs2bPrl5eUlHDttdfWb+NrX/saI0aMSPhdwRI2pXM8tXZK57ufXc5Dc9fw3o/Hxr8oETlsmtI5cQ53SueUHvYBjfmLiMSS2uGvIX8RkZhSO/zRef4iyaY9DDW3N615TVM6/DWrp0hyyc3NZevWrfoDEEfuztatW8nNzT2s56X82T7q+oskj6KiItavX8+R3J1PDpSbm0tRUdFhPSelw1+n+Yskl6ysLPr37x92GUKKD/uIiEhsKR/+ushLRORAKR3+GvUREYktkbdxfMDMNpvZ0gbLupnZS2a2MvjaNVH7r6OTCkREDpTInv+DwPlNlk0GXnH344FXgu8TRgd8RURiS+Q9fF8HtjVZfAnwUPD4IeDSRO2/vo5E70BEpB1q6zH/nu6+ESD4elRzK5rZJDMrM7Oy1p4TrIu8RERiS9oDvu4+zd1L3b20sLDwSLYTx6pERFJDW4f/JjPrBRB83ZzInWnMX0QktrYO/yeBa4LH1wB/TfQO1e8XETlQIk/1nAHMBU4ws/VmdgMwBfiima0Evhh8nzDq+IuIxJawuX3cfUIzPxqTqH3GrqMt9yYi0j4k7QHfuNCgv4hITKkd/iIiElNKh7/6/SIisaV0+NfRuf4iIo2ldPhryF9EJLaUDv866viLiDSW0uGvuX1ERGJL6fCvo46/iEhjaRH+IiLSWEqHvw74iojEltLhX0eneoqINJbS4a+Ov4hIbCkd/nXU7xcRaSylw19j/iIisaV0+NfRkL+ISGMpHf6mrr+ISEwpHf51XKP+IiKNpEX4i4hIY6GEv5n9s5ktNbNlZvbtRO9PY/4iIo21efib2VDgRuAUYDhwoZkdn5h9JWKrIiLtXxg9/0HAm+5e4e7VwN+Ay0KoQ0QkbYUR/kuBUWbW3cw6AhcAxzRdycwmmVmZmZWVl5e3akea0llEJLY2D393Xw78FHgJeB5YDFTHWG+au5e6e2lhYeER7vOIni4iknJCOeDr7ve7e4m7jwK2ASsTsR+N+YuIxJYZxk7N7Ch332xmfYHLgdMTuT+d5y8i0lgo4Q88ZmbdgSrgFnf/NBE7UcdfRCS2UMLf3c9q2/215d5ERJJfSl/hqzF/EZHYUjr8RUQktpQO/4yg619dq3EfEZGGUjr8syLR5lXX1IZciYhIckmP8FfPX0SkkZQO/8xIdNinSj1/EZFGUjr8s+rDXz1/EZGGUjz8NeYvIhJLSod/Zka0eer5i4g0ltLhn52pMX8RkVhSOvzrev7VtQp/EZGGUjv8dcBXRCSmlA7/7EjdmL96/iIiDaV0+Gcp/EVEYkrp8M/OjDavslrhLyLSUEqHf25WBID9Cn8RkUZSOvxzgp7//iqFv4hIQ6GEv5l9x8yWmdlSM5thZrmJ2E9d+O+rrknE5kVE2q02D38z6wPcCpS6+1AgAoxPxL5y6oZ91PMXEWkkrGGfTKCDmWUCHYENidhJbt2wj3r+IiKNtHn4u/vHwM+BdcBGYIe7v9h0PTObZGZlZlZWXl7eqn1lRjKIZBj71PMXEWkkjGGfrsAlQH+gN5BnZhObrufu09y91N1LCwsLW72/nMwM9fxFRJpoUfib2UAzywken2Nmt5pZQSv3eS6w2t3L3b0KeBz4fCu3dUjR8FfPX0SkoZb2/B8DaszsOOB+or32P7Vyn+uA08yso5kZMAZY3sptHVJuVkQHfEVEmmhp+Ne6ezVwGXCPu38H6NWaHbr7POBRYCHwTlDDtNZsqyVyMjN0qqeISBOZLVyvyswmANcAFwXLslq7U3e/A7ijtc8/HDmZ6vmLiDTV0p7/dcDpwF3uvtrM+gN/TFxZ8ZOTpQO+IiJNtajn7+7vEr0wq+5snXx3n5LIwuIlNzOiUz1FRJpo6dk+r5lZZzPrBiwGppvZ1MSWFh/q+YuIHKilwz5d3H0ncDkw3d1HEj1lM+npVE8RkQO1NPwzzawXcCXwdALribucrIjCX0SkiZaG/4+AF4AP3H2+mQ0AViaurPjJycxgX5WGfUREGmrpAd9HgEcafP8hcEWiioqnnEz1/EVEmmrpAd8iM5tlZpvNbJOZPWZmRYkuLh5yMjPYr56/iEgjLR32mQ48SXQitj7AU8GypJerMX8RkQO0NPwL3X26u1cH/x4EWj/VZhuqO9vH3cMuRUQkabQ0/LeY2UQziwT/JgJbE1lYvORk1d3QRb1/EZE6LQ3/64me5vkJ0RuwjCM65UPSy8kMbuWo8BcRqdei8Hf3de5+sbsXuvtR7n4p0Qu+kl5uXc9fB31FROodyZ28botbFQmknr+IyIGOJPwtblUkUI5u4i4icoAjCf92cfpMXfhrZk8Rkc8c9ApfM9tF7JA3oENCKoqz3Ky6YR/1/EVE6hw0/N09P947NLMTgD83WDQA+IG73xPvfUGDYR/1/EVE6rX0No5x4+4rgGIAM4sAHwOzErW/nCwd8BURaepIxvzjYQzRmULXJmoHn435a9hHRKRO2OE/HpgR6wdmNsnMysysrLy8vNU76Jgd7fnvqVT4i4jUCS38zSwbuJgGU0U35O7T3L3U3UsLC1s/jVD3TjkAbN29v9XbEBFJNWH2/McCC919UyJ3kpcdoUNWhPJdCn8RkTphhv8EmhnyiSczo0d+NlvU8xcRqRdK+JtZR+CLwONtsb/CTjmUK/xFROqFEv7uXuHu3d19R1vsrzA/R8M+IiINhH22T5voXdCBj7btpba2XcxIISKScGkR/oOO7szeqhrWbqsIuxQRkaSQHuHfqzMAyzfuDLkSEZHkkBbhf3zPTkQyTOEvIhJIi/DPzYowsDBP4S8iEkiL8AcY3Ksziz7aoYO+IiKkUfifeXwhW3bvZ9kG9f5FRNIm/EefUIgZvLw8obNJiIi0C2kT/t075TDimAJefW9z2KWIiIQubcIfYMygnrzz8Q4279wXdikiIqFKq/AfdXx0aug3V28LuRIRkXClVfgP6pVPx+wIC9Yo/EUkvaVV+GdGMhhW1IWF67aHXYqISKjSKvwBSvp25d2NO9mr2zqKSBpLu/AfeWxXamqdRR+p9y8i6Sstw98M5mvcX0TSWNqFf0HHbLIjGUx96f2wSxERCU1Yt3EsMLNHzew9M1tuZqe35f6LjykAYOe+qrbcrYhI0gir5/9L4Hl3PxEYDixvy53/2/knADBbV/uKSJpq8/A3s87AKOB+AHevdPc2Pfo64piuHN05l6cWb2zL3YqIJI0wev4DgHJgupm9bWa/M7O8tiwgI8O4cFgv/vb+Zj7dU9mWuxYRSQphhH8mUALc6+4jgD3A5KYrmdkkMyszs7Ly8vK4F3HFyCKqapxZb38c922LiCS7MMJ/PbDe3ecF3z9K9I9BI+4+zd1L3b20sLAw7kUM6tWZHp1y+NHT7+KuG7yISHpp8/B390+Aj8zshGDRGODdtq4D4J9O7QvAa+/H/5OFiEgyC+tsn28BD5vZEqAY+EkYRXxz9HH0KejA1Bff1+0dRSSthBL+7r4oGNIZ5u6XuvunYdSRnZnB7V/6HO98vINfvLQijBJEREKRdlf4NnXJ8D4A/Gb2B7roS0TSRtqHf0aG8eQ3zyDD4N8eWRJ2OSIibSLtwx9gWFEBV518DM8v+4RX39MN3kUk9Sn8A//x5cEAXP9gGVt37w+5GhGRxFL4B/JyMrlvYvRygwt/PSfkakREEkvh38D5Q3txUp8ubNyxjx89FcqlByIibULh38QjN0Vnl37gjdVs3rkv5GpERBJD4d9EblaEB64tBeCUn7xCRWV1yBWJiMSfwj+GL5zYk9u++DkALvvNP0KuRkQk/hT+zbh1zPGM6FvAik27eHHZJ2GXIyISVwr/g5g56TS6dMhi0h8W8MaqLWGXIyISNwr/g8jJjPDHG04F4OrfzWOLzv8XkRSh8D+Ek4q68IMLoxeAff7uVzX3v4ikBIV/C1x/Zn+uKj2Gyppa+n/32bDLERE5Ygr/FppyxUn1j8dPmxtiJSIiR07h30Jmxns/Ph+ANz/cxl3P6ApgEWm/FP6HITcrUv8H4Ld/X82z72wMuSIRkdZR+B+m3KwIS+48j4GFeXz7z4uY+8HWsEsSETlsoYS/ma0xs3fMbJGZlYVRw5HonJvFIzd9nmO7deRrD81n/pptYZckInJYwuz5j3b3YncvDbGGVuuWl80fbjiV7p1y+Mp9czl36t90E3gRaTc07HMEju6SyxO3nAHAqs27GfC9ZynfpQvBRCT5hRX+DrxoZgvMbFKsFcxskpmVmVlZeXl5G5fXct3ysll99wUUH1MAwMl3vawrgUUk6YUV/me4ewkwFrjFzEY1XcHdp7l7qbuXFhYWtn2Fh8HMeOKWM/jySb0AKP2vl9m4Y2/IVYmINC+U8Hf3DcHXzcAs4JQw6oi331xdwv9cNRyA0+9+lScXbwi5IhGR2No8/M0sz8zy6x4D5wFL27qORLlsRBH3Xh29F/CtM96m3+RneGaJrgcQkeQSRs+/JzDHzBYDbwHPuPvzIdSRMGNP6sWzt55V//0tf1pIv8nP6GCwiCQNaw+zVJaWlnpZWbu7HAB35+klG/nWjLcbLX/vx+eTmxUJqSoRSRdmtqC50+l1qmcCmRkXDe/NBz+5gHMH9axffuJ/Pk+/yc9w97PLQ6xORNKZev5tqHzXfr7957d5Y9VnU0JEMoxIhjHn30dzVH5uiNWJSKo5WM9f4R+SNVv28N3H32Huh7HnBrpv4ki+OLgnkQxr48pEJFUo/JPY+k8ruPK+uWzYse+Q65Ye25V/Oe8EduytpKBjNgN65DFv9TbOOaGQDlkRIhnGuxt38uAba+jTtQO5WRGu/Xw/9lbWkJsV4eXlm+jXPY8r7v0HlTW13HT2QL52Vn+yMjJYt62Crz4wj137qrl4eG/Gn9KX/j3yKMzPqa9zwdpPeWHZJ2zauZ9Hvn46GRlGZXUtWRHDTH+kRJKNwr+dmL9mG4s/2s7Lyzfx5oftd7K4E4/O571PdsX82SXFvfnhxUPIyDDysjP1yUYkgRT+KaCisppfvPg+989Z3ew6+TmZ3HBWfyqra3l43jp27K06YJ3+PfJYvWUPE0/ry4K121m+cWf9z7IixhnH9eC1FbGn0xjSuzM3nzOQGW+ta3TcIlFO7d+Nk/t1Y922Cn5x5XAqq2vJy8lM+H5FUoXCX9rM1t37mb2inFP7d6OoawfMjFWbd/H2uu2c3K8bX//DAlZsiv2pIB4uHNaLp5ds5MSj8/nDDafyaUUlBR2zdDBd0pLCX5Jeba2TkWFs21PJays207dbR155bzP3z1lNZXVtwvf/T6f25eLhvSk+poDcrAjrtlaweuseThvQjZxMXZMh7ZPCX1KSu7NzbzVla7dR0rcrI378Uv3PThvQLSHHTQYW5rF1TyW3nHMcu/dXM7RPFz7dU8mXhh5NTmYGVTW1mBmdNDwlSUDhLxKorqnl04oqlm7YQX5OJm+tif7hGD/tzQPWPbZ7R9ZurTjifZ54dD6XFPfhipI+bN9bRSTDWF2+hzGDjtJZUpJQCn+RVqqorOapxRu4svQYxt03l1WbdxMJhqcSoU9BB3burWJIn87854WDOfHozvVnRFVW15KZYWToDClpIYW/SBtyd/701jq+P2spFw/vzSc79nFcz078ad66hO3z/CFH899fGUZ2JDpjy859VRR2ytEnizSn8BdJQrW1zqy3P2ZIn84MLOzE+GlvsuKTXZw3uCePv/1x3PeXHclgyhUnMWfVFoYXFTBuZBF5OZnsqKgiNzuDiEUv1tO1F6lD4S/Sjrn7AT34zTv3sX1vFfurarl/zoc8seizGwdlRYyqmvj8vz62e0c+P7AHHbIinNgrnytKiur/OOzeX02GQcdsHdxOVgp/kTRVXVOLAys+2cWFv54DwMhju/JB+W62Vxx4EWC85Odm0qegAx2zI/xy/Ahmzl/HP4/5HNmZGXy0rYKCjlnk52YlbP8SpfAXkRarqKxm7dYKXn1vMxcN682tM99meFEXHpq7NiH7y8/JZNf+6gOW9+3WkStLi7jp7IG8vHwTv351FfdNHMkx3TpSl1s6pnFwCn8Riat9VTWYQU5mBHfn7yu3UL5rP1eMLGLP/mr2VtVw55PLeDpJbmH6pSE9eWHZJgC+PKwXpw/ozpyVW/jhJUMwi55J1SErQre87Po/KHXDbe7O/upacjIzDvnHJtYQXZiSMvzNLAKUAR+7+4UHW1fhL5Jadu2rYtFH2zmpTxcqa2pZuPZTzjy+kN37qnnvk5107ZjN7+as5qnFGw69sRT30ndGcXzP/FY9N1nD/zagFOis8BeRw1Vb63y4ZTedc7OocSczI4MuHbK46NdzGHhUHlOvLOahf6xh0UfbeW7pJ/Tukkthfg6L1+8Iu/QW69IhiwevO5kRfbu26vkHC/9QDtObWRHwZeAu4LYwahCR9i0jwzjuqAN7xC98Z1T946+fPfCwt1tb69SN3BxsCMfdqapxFn20nWFFXfhkxz56FeTWD4Wt2LSLvZU1DOrVmdysCLv3V9MxK8JjC9dzx5PLmP/9c3l/0y6O6pxL7y7RiQd37qvmo20V7K+uZUjvzgm913coPX8zexS4G8gHbo/V8zezScAkgL59+45cuzYxB5tERFJVUt3A3cwuBDa7+4KDrefu09y91N1LCwsL26g6EZH00ObhD5wBXGxma4CZwBfM7I8h1CEikrbaPPzd/bvuXuTu/YDxwKvuPrGt6xARSWdh9PxFRCRkoU7K4e6vAa+FWYOISDpSz19EJA0p/EVE0pDCX0QkDSn8RUTSkMJfRCQNKfxFRNKQwl9EJA0p/EVE0lC7uJOXmZUDrZ3WswewJY7ltAdqc3pItzanW3vhyNt8rLvHnBmzXYT/kTCzsuamNE1VanN6SLc2p1t7IbFt1rCPiEgaUviLiKShdAj/aWEXEAK1OT2kW5vTrb2QwDan/Ji/iIgcKB16/iIi0oTCX0QkDaV0+JvZ+Wa2wsxWmdnksOs5Ema2xszeMbNFZlYWLOtmZi+Z2crga9dguZnZr4J2LzGzkgbbuSZYf6WZXRNWe2IxswfMbLOZLW2wLG5tNLORwWu4KniutW0LD9RMm+80s4+D93qRmV3Q4GffDepfYWZfarA85u+6mfU3s3nBa/FnM8tuu9YdyMyOMbPZZrbczJaZ2T8Hy1P2fT5Im8N9n909Jf8BEeADYACQDSwGBodd1xG0Zw3Qo8my/wYmB48nAz8NHl8APAcYcBowL1jeDfgw+No1eNw17LY1aM8ooARYmog2Am8BpwfPeQ4Ym6RtvhO4Pca6g4Pf4xygf/D7HTnY7zrwF2B88Pg+4OaQ29sLKAke5wPvB+1K2ff5IG0O9X1O5Z7/KcAqd//Q3SuBmcAlIdcUb5cADwWPHwIubbD89x71JlBgZr2ALwEvufs2d/8UeAk4v62Lbo67vw5sa7I4Lm0MftbZ3ed69H/I7xtsKzTNtLk5lwAz3X2/u68GVhH9PY/5ux70eL8APBo8v+HrFwp33+juC4PHu4DlQB9S+H0+SJub0ybvcyqHfx/gowbfr+fgL3iyc+BFM1tgZpOCZT3dfSNEf8GAo4LlzbW9Pb4m8Wpjn+Bx0+XJ6pvBMMcDdUMgHH6buwPb3b26yfKkYGb9gBHAPNLkfW7SZgjxfU7l8I81zteez2s9w91LgLHALWY26iDrNtf2VHpNDreN7ant9wIDgWJgI/CLYHnKtNnMOgGPAd92950HWzXGslRpc6jvcyqH/3rgmAbfFwEbQqrliLn7huDrZmAW0Y+Am4KPuQRfNwerN9f29viaxKuN64PHTZcnHXff5O417l4L/Jboew2H3+YtRIdJMpssD5WZZRENwYfd/fFgcUq/z7HaHPb7nMrhPx84PjgKng2MB54MuaZWMbM8M8uvewycBywl2p66sxyuAf4aPH4S+GpwpsRpwI7go/QLwHlm1jX4iHlesCyZxaWNwc92mdlpwRjpVxtsK6nUhWDgMqLvNUTbPN7McsysP3A80YObMX/XgzHv2cC44PkNX79QBK/9/cByd5/a4Ecp+z431+bQ3+cwj4In+h/RMwXeJ3qE/Pth13ME7RhA9Mj+YmBZXVuIjvW9AqwMvnYLlhvwm6Dd7wClDbZ1PdEDSKuA68JuW5N2ziD68beKaC/nhni2ESgN/oN9APwvwRXuSdjmPwRtWhIEQa8G638/qH8FDc5iae53PfjdeSt4LR4BckJu75lEhySWAIuCfxek8vt8kDaH+j5regcRkTSUysM+IiLSDIW/iEgaUviLiKQhhb+ISBpS+IuIpCGFvyQFM6sJZjZcbGYLzezzh1i/wMy+0YLtvmZmaXXT70MxswfNbNyh15RUpvCXZLHX3YvdfTjwXeDuQ6xfABwy/MPS4GpLkaSk8Jdk1Bn4FKLzoZjZK8GngXfMrG5m1inAwCxmWigAAAOsSURBVODTws+Cdf8tWGexmU1psL2vmNlbZva+mZ0VrBsxs5+Z2fxgYq2vB8t7mdnrwXaX1q3fkEXvrfDTYJtvmdlxwfIHzWyqmc0GfmrROeqfCLb/ppkNa9Cm6UGtS8zsimD5eWY2N2jrI8FcMJjZFDN7N1j358GyrwT1LTaz1w/RJjOz/w228QyfTZom6SzsKxz1T//cHaCG6JWP7wE7gJHB8kyiU/QC9CB6BaMB/Wg8B/5Y4B9Ax+D7uitEXwN+ETy+AHg5eDwJ+I/gcQ5QRnTu9H/hsyuoI0B+jFrXNFjnq8DTweMHgaeBSPD9r4E7gsdfABYFj38K3NNge12Dtr0O5AXL/h34AdH56lfw2f22C4Kv7wB9mixrrk2XE53yOAL0BrYD48J+z/Uv3H/6aCrJYq+7FwOY2enA781sKNGg/4lFZzGtJTpVbc8Yzz8XmO7uFQDu3nCO/LrJwxYQ/aMB0blghjUY++5CdA6V+cADFp2I6wl3X9RMvTMafP2fBssfcfea4PGZwBVBPa+aWXcz6xLUOr7uCe7+qZldSPQmHm9Ep4IhG5gL7AT2Ab8Leu1PB097A3jQzP7SoH3NtWkUMCOoa4OZvdpMmySNKPwl6bj7XDPrARQS7a0XEv0kUGVma4DcGE8zmp/Gdn/wtYbPfucN+Ja7HzCxXfCH5svAH8zsZ+7++1hlNvN4T5OaYj0vVq1G9OYkE2LUcwowhugfjG8CX3D3m8zs1KDORWZW3FybLHp7QM3jIo1ozF+SjpmdSHSIYivR3uvmIPhHA8cGq+0ieku8Oi8C15tZx2Ab3Q6xmxeAm4MePmb2OYvOnnpssL/fEp2JsaSZ51/V4OvcZtZ5Hbg62P45wBaPzuP+ItEQr2tvV+BN4IwGxw86BjV1Arq4+7PAt4nO/Y6ZDXT3ee7+A6JT+h7TXJuCOsYHxwR6AaMP8dpIGlDPX5JFBzOrG2Ix4Bp3rzGzh4GnLHrT+rpjArj7VjN7w6I3Pn/O3f816P2WmVkl8CzwvYPs73dEh4AWWnScpZzore/OAf7VzKqA3UTH9GPJMbN5RDtQB/TWA3cC081sCVDBZ1MW/xfwm6D2GuCH7v64mV0LzDCznGC9/yD6R+6vZpYbvC7fCX72MzM7Plj2CtEZX5c006ZZRI85vEN0Rsi/HeR1kTShWT1FDlMw9FTq7lvCrkWktTTsIyKShtTzFxFJQ+r5i4ikIYW/iEgaUviLiKQhhb+ISBpS+IuIpKH/AyP9vv2PL1tVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot_losses()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>3.981084</td>\n",
       "      <td>3.761016</td>\n",
       "      <td>0.309128</td>\n",
       "      <td>32:07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, 1e-2, moms=(0.8,0.7))\n",
    "learn.save('heb_spm_2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3hUVfrA8e9LEhIgoQcEAoQqNYQQKYt0pNpQVFBWsbGLuhbU3aCuIjasy7o/1y42FBEXGyCCglhoASH0ogQIIIReAyQ5vz/mZphkZpIJycxkct/P8+Thzr1n7rxnJuSde84954gxBqWUUvZVIdgBKKWUCi5NBEopZXOaCJRSyuY0ESillM1pIlBKKZsLD3YAxVW7dm0THx8f7DCUUiqkrFixYr8xJtbTsZBLBPHx8aSmpgY7DKWUCikist3bMW0aUkopm9NEoJRSNqeJQCmlbC7k+giUUuXH2bNnycjIICsrK9ihlBtRUVHExcURERHh83M0ESilgiYjI4OYmBji4+MRkWCHE/KMMRw4cICMjAyaNGni8/O0aUgpFTRZWVnUqlVLk0ApERFq1apV7CssTQRKqaDSJFC6zuf91ESgAm7ljkOs3XUk2GEopSzaR6AC6t2ftzHhq/UA3Nu/BSfP5PDgwAuJCMv/nWTfsSyqVAynSqT7r+jS3w9wOjuXni09DpJUyicHDhygX79+APzxxx+EhYURG+v4nVq2bBkVK1Ys8hw333wzKSkpXHjhhX6N1d8k1BamSU5ONjqyuGxbsf0QS7cd4I7ezYlPmQXAX3o15fUffvf6nJ4tY6lXNYqUwa2oUaUi8SmziK9VmYUP9nErm3fO9ElD/VMBFTAbNmygdevWwQ6DCRMmEB0dzQMPPJBvvzEGYwwVKoRW44mn91VEVhhjkj2VD63aqZBw9au/8Nw3m3ho5hrnvsKSAMCizZl8krqTkW8uITfX8eUk/cBJ5/Hjp7NZ/NsBEid+65+glbJs3bqVdu3a8de//pWkpCT27NnDmDFjSE5Opm3btkycONFZ9uKLL2bVqlVkZ2dTvXp1UlJS6NChA926dWPfvn1BrEXxaNOQKpHL/vMTN3ZrzPBOcRw+eZZqlc7du/zR0h3FPt/GP47x/uJ05+OzOblEhFVg/P/W8NXq3fnKHjmV//VUaHv8q3Ws3320VM/Zpn5VHrusbbGft379eqZMmcJrr70GwKRJk6hZsybZ2dn06dOH4cOH06ZNm3zPOXLkCL169WLSpEmMGzeOd955h5SUlFKph79pIlDn7dCJM6zZdYQHZ6Tx4Iy0UjtvXh8CQIuH53gt1+Hxb3nxmg5c3SkOONdkJAJjejRlYLsLSGpUo9TiUvbRrFkzLrroIufjjz/+mLfffpvs7Gx2797N+vXr3RJBpUqVGDx4MACdOnXixx9/DGjMJeH3RCAiYUAqsMsYc2mBY38F7gRygOPAGGPMevezqLKo4xPzgh0C93+62pkI8hgDry/6ndcX/U6HhtVZvfMw254ZorcplnHn883dX6pUqeLc3rJlC//+979ZtmwZ1atXZ9SoUR7v03ftXA4LCyM7OzsgsZaGQPQR3ANs8HLsI2NMe2NMIvAc8FIA4lFBNn9cz1I9X06ucV4NFLR652EAdh0+xZ/fXsprP/xWqq+tyr+jR48SExND1apV2bNnD3Pnzg12SKXOr4lAROKAocBbno4bY1wbBKsAoXULkzovsdFR3Nw9HoDxg1vRo0XtEp1v4ORFRZaZnprBj1v2M2nORs7m5Jbo9ZS9JCUl0aZNG9q1a8ftt99O9+7dgx1SqfPr7aMiMgN4BogBHijYNGSVuRMYB1QE+hpjtngoMwYYA9CoUaNO27d7XV9B+dlnKzLo0aI2nZ/+zmuZmMhwOjauwaLNmdSsUpGV/7wEONeGv/GJQURFhDnLZ53NYe/RLBrVrEyT8bOd++NqVCLj0Kl8525Suwrb9p9wPm51QQwb/zhWrDp8e19P9hzJIm3nYf7Wr0WxnqtKV1m5fbS8Ke7to37rIxCRS4F9xpgVItLbWzljzCvAKyJyPfAIcJOHMm8Ab4BjHIF/IlZFOXjiDPd/urrIctd3bcT4wa1Znn6Q9g2qOfeLONrvXZMAOB43ruVok932zBCajJ9N5yY1mf6Xbs7k0bxONFNGX0RcjUoYA00fciSM4iYBgNNnc7npnWUADGp3AS3qxhT7HEqVJ/5sGuoOXC4i6cA0oK+IfFhI+WnAlX6MR5XQ0VNnvR6bfF2ic/u+/i0BuCi+Zr4/+tueGVrkIDARYd59PXlntOOOjchwx6/oF3d2p2HNyogIFSoI08Z0Pe96DH/tF+f283M3Obdn/ppBfMos9h3TKZGVvfgtERhjxhtj4owx8cAI4HtjzCjXMiLiel0+FHBrFlLBl5trOHTiDL1fWOi1TOt6VZ3bBb/xF1eLujFEW1NLfHR7F27uHu821UTXprU8Pnf84FZc3qE+VybW93r+09nn+gg27XVcUfzy237u+8RxtbP094Mlil+pUBPwcQQiMhFINcZ8CdwlIv2Bs8AhPDQLqeB7YMZq/rdyV6FlIsLEL1M+dGpck06Na3o89vqfO/GXD1Y4Hxd8/c9X7S74FDfbrdHLU10Gv2XnameyspeAJAJjzEJgobX9qMv+ewLx+qr4jDGcOJNDdGQ4y7Z5/4bcsVF1erWMpUntKl7L+MvAthcUq/zTw9rnm/YiT5en57P36Gnn4zPWFUNq+kF2HjrJsI5xbs9RqjzRkcXKo+mpO/nHZ2t488Zkjpx07xuoGhVOvWqVmP6Xbm4zhwbS6kcHMOmbjTw9rF2RZVvUjfa4f+/R0wzr2ICZvzquetbuOsoF1TKdHcrtG1SjeR3tUFbll046p9wcOXWWf3zm+OZ8+/upHDudf4Rk+qShpE0YyNz7egY1CQBUqxzBM1e19zhqOH2So3N69WMDeOvGZDoVMt1E2/rn+jg+WLLdmQQADntIhKp86N27t9sAscmTJ3PHHXd4fU50tOMLxe7duxk+fLjX8xY1S/LkyZM5efLcxIpDhgzh8OHDvoZeqjQRqHwW/3aADo+Xrxk+q1WKoH+bulSoIDw/PMG5/84+zZzbT87yNvjdMfPprzsOcefUlX6NUwXeyJEjmTZtWr5906ZNY+TIkUU+t379+syYMeO8X7tgIpg9ezbVq1c/7/OVhCYC5bR13zFGvb200DLxtSoHKBr/uKyD426ii+Jr8ODAVj49Z/SU5Qz77y/MWrPH61QWKjQNHz6cr7/+mtOnHX1E6enp7N69m8TERPr160dSUhLt27fniy++cHtueno67do5miRPnTrFiBEjSEhI4LrrruPUqXMDIceOHeucwvqxxx4D4OWXX2b37t306dOHPn0ca27Ex8ezf/9+AF566SXatWtHu3btmDx5svP1Wrduze23307btm0ZMGBAvtcpCe0jsLkX5m7ix637GZZYP9+sn950aeL5ts1QERURxmdjuznb/G/o0ijfHUO+2LrvOM3reO5vUCUwJwX+cO/ML5EL2sPgSV4P16pVi86dO/PNN99wxRVXMG3aNK677joqVarEzJkzqVq1Kvv376dr165cfvnlXicufPXVV6lcuTJpaWmkpaWRlJTkPPbUU09Rs2ZNcnJy6NevH2lpadx999289NJLLFiwgNq180+xsmLFCqZMmcLSpUsxxtClSxd69epFjRo12LJlCx9//DFvvvkm1157LZ999hmjRo0qGE6x6RWBzf3fgq2s3nnYpyQw6+6LefyKsjND5Pnq1Limcx2DDXuKP/99/5d+4PDJM+w9qgPPygPX5qG8ZiFjDA899BAJCQn079+fXbt2sXfvXq/nWLRokfMPckJCAgkJ55ogp0+fTlJSEh07dmTdunWsX1/4/7WffvqJYcOGUaVKFaKjo7nqqqucU1o3adKExETH4M1OnTqRnp5ekqo76RWB8skzV7Wnbf1qRRcMMSt3nOuca12vKpnHsth//AwADw68MN/IY1eJEx1TcK94pD+1oiP9H6gdFPLN3Z+uvPJKxo0bx8qVKzl16hRJSUm8++67ZGZmsmLFCiIiIoiPj/c49bQrT1cL27Zt44UXXmD58uXUqFGD0aNHF3mewuZ/i4w897sWFhZWak1DekWgvGpZN5pxl7SkeZ1orkpqEOxw/O69Wy5iwQO9WfFIf9InDeXOPs25oUujQp/zwRKdADHURUdH07t3b2655RZnJ/GRI0eoU6cOERERLFiwgKImuuzZsydTp04FYO3ataSlORZqOnr0KFWqVKFatWrs3buXOXPOLbQUExPDsWPuc2X17NmTzz//nJMnT3LixAlmzpxJjx49Squ6HukVgY3tK6Rp46d/9KFB9UqICHeX4xk6XccP1ImJcjv+1LD2hfYhTJ6/hbb1q3FJm7rsO5rFqbM5zgn0VOgYOXIkV111lbOJ6IYbbuCyyy4jOTmZxMREWrUq/MaCsWPHcvPNN5OQkEBiYiKdO3cGoEOHDnTs2JG2bdvStGnTfFNYjxkzhsGDB1OvXj0WLFjg3J+UlMTo0aOd57jtttvo2LFjqTUDeeLXaaj9ITk52RR1f64q2vHT2fR/8Qf+8JAMGlSvxM8pfYMQVXB8tHQHV3dqQGS45zmSfLlT6Pv7e9H3xR8A96kulHc6DbV/FHcaam0asql2j831mAQAqkSWbNK4UHN9l0Zek4CrL+70viBJXhJQKhRp05ByM7hdvWCHUKakTxpKTq4hrIKueazKJ70isKGCzYEjLmoIwIe3dmH+uJ7c27/89gmcr7wk8K/rOhRZNjsnF2MMM3/NcE5gp7wLtebpsu583k+9IrChN3/8Pd/jSVcnMOnqBC+llathHeO4MrEB32/cx63vee6rum/6ar5a7ZgC+38rd/HBrV3Yf/w063YfpVfL2ECGW+ZFRUVx4MABatWq5XWwlvKdMYYDBw4QFeV+40NhNBHY0NOzNzq3OzQMztwmoUxE6Ne6rtfjeUkA4Mct+1n6+wGue2MJAP+9IYkh7bXpLU9cXBwZGRlkZmYGO5RyIyoqiri44k2drncN2cimP44hAgP+tci5b/3EgVSuqN8Hzsfhk2d456dtvPz91mI9T+8qUsEQlMXrVdny1erd/O3jX/Pt69CwuiaBEqheuSLjBlzImF7NOHTiDC98u4kvfFgVTamyRjuLbaJgEgD4/I4/BSGS8ic6MpyGNSsz+bpEn8pnnc3xc0RKFY8mAhs4eSbbbd+Mv3bTzrlS5uv7OfjfP3I0Sxe7UWWHJoJybufBk/n6BPIkx3teEF7537b9J0iY4L74z5/fXsq46auCEJGyO00E5VyP5xaQcah0ZihURXvhmnPjDIrTKfz9xr38uGU//1u5yx9hKVUoTQRKlaIrE+sjAk9e2c75GCCuRiWm3tYlX9n4lFnOeYxueffcnXC/ZR4PULRKOegtIzaS+kh/kp+cH+wwyrXwsApse+bclcDz13SgUsVw7h/Qktpe1i2YlbYn3+N+L/5As9gqPHlle7o1C+0V4VRo0CuCcmjPkVN8scq9iaF2dCRPDWvH/HE9gxCVPUWEVeCZq9p7TQIAd3600m3fb5knGPnmEn+GppSTXhGUQ92e+R6AzGOn3Y7d0KVxoMNRLuaP60n/l9w775UKJr0iKMeenLXBud1CF1svE5rXieHb+/SKTJUtfk8EIhImIr+KyNcejo0TkfUikiYi34mIfl31k3njegU7BGVpWTfG4/7GtSoHOBKlHAJxRXAPsMHLsV+BZGNMAjADeC4A8ZRrB467NwfVraqLq5c1ax8f6LbvxGn3gX/xKbN4ZcFW/vn5WuJTZnHwxBmOeyinVEn4NRGISBwwFHjL03FjzAJjzEnr4RKgeFPmqXxOnsmmk4e7gt69uXMQolGFiY4MZ2Tnhvn2je3d3GPZ5+du4oMljsXTk56YR7vH5vo9PmUv/r4imAz8HfBldY5bgTmeDojIGBFJFZFUna7Wu1cX/ua275MxXWldr2oQolFFeeyytkwb09X5+NaLmwQxGmVnfrtrSEQuBfYZY1aISO8iyo4CkgGPDdnGmDeAN8AxDXUph1pufXNvD1pdoEmgrIqKCKNr01q8NqoTp846mns+G9uNszmGOWv28N7i7V6f+695m7nvkpaBClWVc/68fbQ7cLmIDAGigKoi8qExZpRrIRHpDzwM9DLGuDdwK59knc3hPy7z4jepXUWTQIgY1O4C53anxo45oOpWjSo0Efz7uy3c27+FThyoSoXfmoaMMeONMXHGmHhgBPC9hyTQEXgduNwYs89fsdjBjBUZ+R5/WGA6AxVaKlcMK7JM0hPzAhCJsoOAjyMQkYkicrn18HkgGvhURFaJyJeBjqc8yM7J5ZHP1+bb16B6pSBFo0pD3arn1py9zUvfwaGTZ3Xhd1UqAjKy2BizEFhobT/qsr9/IF6/vDuWlf92wu/u1zED5UHe7KVZZ3N466dtHsuczs4lKqLoqwelCqMji8uBUwVWvGoWq6OIy5OoiDDSJw2lWqUIt2M/bNa76FTJaSIoB75OO7dOri6MXn4dOeW+qtlXq3WNZFVymgjKgadnbwx2CCqAHhrSij93dczG8nXaHk5n53D3x7+ybNtBdh48WcSzlXKns4+GuIMnzji3P7hVRxCXZ3ViItl37DSjujZm4aZM52jjCx/5BoAvrauDbc8M0dtKVbFoIghRx7LOciwrm7EfrnDu69S4RhAjUv72yV+68f3GfVSuGM5gl7EHBa3ddZTwMNER5cpnmghCVHsPi59XrqgfZ3nWpHYV5zQUhX3jH/bfn8nOddxWqn1GyhfaRxBi9hw5xXu/pAc7DFWG5SUBgE+W7+Bsji9TfSk700QQIg6eOMOqnYf589vLeOzLdW7Hr+mkE7faTcdG1Yss84/P1tDi4Tke7zhSKo+E2sjE5ORkk5qaGuwwAi4+ZVahxzdMHEQlH6YlUOXLdxv2cut7vv1/2PLUYCLC9LufXYnICmNMsqdj+ltRTmgSsKd+res6t4vqD2jx8By9vVR5pImgHFj+sM7UoXyz8Y9jwQ5BlUF6m0kIG9W1ER8u2UFsjC5FaWfrHh+Irw28p7Nz2Hcsi6pRETpHkXLSRBCiNj85mIgw4bHL2gY7FBVkVSLP/Td+68Zkbns/lVeuT+JsTi73frIqX9m7PvrVua23lqo82jQUIprWruLc3vjEICqGV0BEtPNP5dO/TV3SJw1laEI9ruzYgDn39GBg27oey7707SYyDp0kPmUW89bvDXCkqizRvyJlzI4D5zrzft66n0e/WMsfR7I4cebcVNN6Sa981bpeVV69oZPHYy9/v5WLn10AwO3vp/Lhku35pixR9qGJoAz5Zu0eej6/gIWbHIu13fDWUt5fvJ2uz3zH3qOnqR1dkUUP9glylCrUVKggPDCg6PWNH/l8LT2fWxCAiFRZo4mgDEnLOALA6CnLPQ4A2n/8DI1qVQ50WKocuKtvC5/KHT/tuPL8Zet+pi3b4c+QVBmiiaAMcf3j3+Fx97mEel8YG8hwVDnjy1UBgDGG699aSsr/1vD6D7/5OSpVFmgiKEOmLi38G9ioLo0DFIkqj65JbgjABS7rIXsyPXWnc/uZObrWhR1oIggh/dt4vvtDKV/UrRrFc8MT+OpvF7Nh4iCv5f7x2Zp8jz8q4guKCn2aCEJEsq41oErBtckNiY2JpFJFxzrIvowleGjmGpanHwxAdCpYdEBZkJ06k0PrR7+hemX3hckB/nVdBwa1rUd4mK44pfzj2uQ4pqdmFFrmmtcW89dezUgZ3CpAUalA0iuCINt95BQAh096niZ4WMc4KlUM04Fjym+eG96B+eN68re+zZ37OjR0n+L6tQIdx8YYcnNDa/Zi5Zn+dQmyt3/a5rZvy1ODAZg2pmugw1E21bxODOMuOXdX0eqdh/m/6zu6lTtkDTg7k51Lk/GzafrQ7IDFqPxHm4aCbPFvB9z2RYRV0HlgVMCJCH/r25z/fL8VgEsT6uebmwig4xPzqF45It8VrDGm0KUzVdmnVwRBsumPY3yxahfb9p/It991TiGlAu3OPo7mofFWX8Dsu3u4lSnYjLnkd+1IDnV+vyIQkTAgFdhljLm0wLGewGQgARhhjJnh73jKioGTF3ncP/se9/94SgVKVERYvqvR6Mii/0SMfHOJXsGGuEBcEdwDbPBybAcwGvgoAHGUaVcnxdGjRW2dUE6VKbVjKvpUruCVrQotfk0EIhIHDAXe8nTcGJNujEkDcv0ZRyh48doOfHBrl2CHoVQ+lSuGE16h6Pb/x79aF4BolL/4+4pgMvB39A+9UiFr69NDPDb9VHZZJ3vhpkzaT5gLOO4oOnxSp7MOJX5LBCJyKbDPGLOiFM41RkRSRSQ1MzOzFKILLmP03msVuv4xqBVTb+vCmgkDaewyG+6xLMfMpS0fmUPixHk6xiCE+POKoDtwuYikA9OAviLy4fmcyBjzhjEm2RiTHBsbujNw/rRlP//8fC1HT2Xn2+9tBSmlypKfU/qy9KF+jO3djO7NaxNWQVhwf+98Zc5kn7v4b/rQbPYdzQpwlOp8+C0RGGPGG2PijDHxwAjge2PMKH+9XlFyc43HOf79/ZrxKbNYtNlxFTPq7aV8sGQ7HSbmn2K6ZhXfOuSUCqYG1StRt8DMpRUK9B9c9p+f8j0e9fZSvQIOAQEfRyAiE0Xkcmv7IhHJAK4BXhcRv/U4PTlrAx0e/5ZTZ3L89RJA/mafSd84pvC98Z1lrN552OtzKkXouD5VPmzaeyzf4817j/Pz1gOcOJ1NjjYVlVkBSQTGmIV5YwiMMY8aY760tpdbVw1VjDG1jDFt/RVD5vHTAJzN9V+/9aY/jtFk/Gze+vF3jmadJS3j3B//K1752a181ShHAoiM0HF9KnSte3xgocdzjKHtY3OZ8KXeWVRW2eYvUIe4an5/jZe/3wI4rj4SJnxb5IjLv/Rqxt39WnBXn+aFllOqLKsSGc6cAgMhr+kU59xesNGxBvcHS7YDsOT3A3y77o/ABaiKZLs2iYQJ39KpcQ0+G/unUj/3rLQ9PpdNGdyKv/ZqVuoxKBUMuQX6AUZ0bsSnKxxTW7/7S7pzf3ZOLiPeWAKgo5HLENskAtdJsVZsPxS0ODY+MUhHD6typ1lsdL7HkeGeGxtemrfZuR2fMovXRiUxqF09v8amimabpqHCnMnO5fGv1hU5CObT1J3sPnzK+Xjmrxn5OoEr+rBmgCYBVR5Fhlegossf/+Z1oml1QYxbuf8uzL+mwXPfbPJ7bKpotkkEy7flb69/Y9FvvGr9Un6dtpspP6fz5CxvUyLBkZNneXBGGre+lwrAe7+kc98nq7nilZ+JT5lFxqGT3PQnXVxe2ZOIsPnJwc7HURFhfHNvT/7Ss2mhz8vWO4nKBNs0Da3bcyTf46dnO27tvL5LI8ZNXw3AjBUZvHBNB7fnZp3N4Y6PHAOkN+w5CsBjBe6AuPjZBaUes1KhZsroi6hTNdL5ePyQ1ry+6Hev5XccPBmIsFQRbJMITp3xfNvoul35E0TmsdNUqxTBDW8t4aPbuxIRVoFW//wmX5ntB4o30+InY7oyZ+0fNKpZuejCSoWwPq3quO1LnzSUrLM5bv+PVNnhUyIQkWZAhjHmtIj0xrF+wPvGGO+jpMoYbwsoHSqwyMZFT80nqVF1Vu44TIuH5/Dj3/u4PafX8wuL9dpdmtaiS9NaxXqOUuVJYX1jW/Yeo2HNykRFhPF12m6SGtWgfvVKAYxOiS/Dv0VkFZAMxANzgS+BC40xQ/wanQfJyckmNTW1+M97ch77jwdmRsSpt3WhWWw0e49mYYBEDwuBK2U3x09n0+6xuUWWqxMTybKH+wcgInsRkRXGmGRPx3xtGso1xmSLyDBgsjHmPyLya5HPKlP8s6Zqg+qV2OVyJxFA9+a1AbigWpSnpyhlS9GR4Xx118WcOJNN1tkcRk9Z7rHcvmOneX7uRlbvPMKHt+kaHYHgayI4KyIjgZuAy6x9Ef4JyT+qRoWz35pmojT9nNKX935Jd3YeTxvTtdRfQ6nyor3LCP+YqHDn1NUFvbLAcUffyTPZhFfIf2uqKn2+vrs3A92Ap4wx20SkCXBeU0oHS7XKjrz18e1dWfhAb7fjyx7uV+Q5tjw12OP+m/4Uz/VdGrHyn5fQVfsClPLJDw+6978V1ObRubR8ZE4AorE3nxKBMWa9MeZuY8zHIlIDiDHGTPJzbKUqb5GMShXDiK9dhfRJQ3lueILzeJ2YKLZ6+UOfJ6LAgDHXDuinh7XX6aSVKobi/n/JzTX8uiN4swKUZz4lAhFZKCJVRaQmsBqYIiIv+Te00pVjdYq7rr96bXJDwisIN3Rp5Djmw8hgVy+P6Fh6ASqlvMo4dJLXF/3OsP/+woOfrg52OOWOr30E1YwxR0XkNmCKMeYxEUnzZ2ClrX2DaqzddZTqlfN3bWx92vuNT8M6NuCJK9vx9erdDLdmU0yfNJR56/cyK203lyboHClKlcTnd3Zn9po9PDSkNWeyc9m67zhDXv7RrZzrgM1PV2TwvIeBn+r8+ZoIwkWkHnAt8LAf4/GbCZe35frOjYmr4fugrn9dlwg4ZlJ0dUmbulzSRpeXVKqkEhtWd95eXTG8Ahd6mJ/Ik8nzN3NXn+bFvopXnvn6Lk7EMX7gN2PMchFpCmzxX1ilLzI8LN8dC958c69jXvVZd1/s75CUUgWEVfDtNu/J87fQ/OE5vFnI9BXKdz4NKCtLzndAmVIqNLy/OJ1Hv/B9NTNd18A3hQ0o87WzOE5EZorIPhHZKyKfiUhc0c9USqniubFbfLH+uN8/fTXxKbNYt/uI27H4lFnEp8wqzfDKJV+bhqbgmFaiPtAA+Mrap5RSfrHyn5eQNmEAayYMKLTcZysdK6ENffmnfOuEH/DDANLyytdEEGuMmWKMybZ+3gVi/RiXUsrmalapSNWoCGKiIkifNNSnq4Q7pq4kN9eQcegknZ6cH4AoywdfE8F+ERklImHWzyjggD8DU0qpgga2LfxuvdxcwzWvL3ZbH+TQicBMOBmqfE0Et+C4dfQPYA8wHMe0E0opFTBDE+oXenz3kSyPa5I/+7Zb6k8AABJZSURBVM1G4lNmcdM7y9yOZR7zrQnp3Z+30eO5730LNMT4OsXEDmPM5caYWGNMHWPMlcBVfo5NKaXySYw7vyndpy3fCcAPmzPZezTLuf/dn7dx0VPzWVZgKVtPJny1np0HT5FTDpfXLMlojHGlFoVSSvmgYc1KjLukJbd0b0Jfl9XQinOX0d9npPH+4nSyc3KZ8NV6AK59fTEA2Tm5ZOd4Xs0wz6ItmcUPvIwryVKV/pngXymlvBAR7u7XAgBjDE3Gzy72OX7YnMkPmzOZtmyn27HmDztmOi0ssdSoXP4mlyzJFUH5uz5SSoUMESnybqLbLm7i9dj6PUe9Hvst87jbvpgox/fmE6c9r6EQygpNBCJyTESOevg5hmNMQZGsu4x+FZGvPRyLFJFPRGSriCwVkfjzqoVSSlnWPj6QERc15OPbuzJ+SGt6tKjt0/PW7z6XGPq9+AMzVmQQnzKLrLM5AM5FdMpj01ChicAYE2OMqerhJ8YY42uz0j3ABi/HbgUOGWOaA/8CnvU9dKWUOmf23T24p18LoiPDmXR1At2a1SKsgtCrpW9DngrOevqANd31iu2H8o1O/nbd3kLP8+gXa5m9Zk8xow8uv07dZ01DMRR4y0uRK4D3rO0ZQD8R0b4HpVSxtalflfsuaem2/1aX5qGp57EG8g1vLc33eNv+E2QcOslHS3e4jV7etv8E7y/ezh1TV7qdxxjDqwt/46yXzuh9R7MI1txv/p7DdTLwd8BbN3wDYCeAMSYbOAK4rfUoImNEJFVEUjMzy99lmVLKf1z7Ei6Kr+l2vM+FxZ8k4eJnF/DQzDV0enI+J05nc9t7qew8eDLfranxKbPYefCk83GT8bN59puNtHg4/9KbazKO0OXp+XR++jv+8VlwlnnxWyIQkUuBfcaYFYUV87DPLSUaY94wxiQbY5JjY3VmC6XU+akYXoHZd/dg/riezsdv33RRic7Z9rG5zN+wl7FTVzDijSX5jvV4bkGRk95d9n8/sfeo48piempGiWI5X/68IugOXC4i6cA0oK+IFFzwPgNoCCAi4UA1oOiRHUopdZ7a1K9K8zoxpE8ayuYnB1OhwBoIL3pZ/ayo6S3W7vJ+F9JTs9a77Vu18zAHPUx9EZ8yy3llsWL7ITb+cZRv1/3B1KXbC339kijJOIJCGWPGA+MBRKQ38IAxZlSBYl8CNwGLcUxb8b0JtQUSlFIhb+U/L2H0lGW8dG0HVu447LFMrehImtSuwrb9J4p9/jd/3Jbv8fBXfyHVw1QYeaYu3cG4S1py9au/5Nt/Q5fGxX5tXwR8nTcRmSgil1sP3wZqichWHCOVUwIdj1JK1axSkS/vupjmdWLo1LiGc/84l87nShFh59XZ7ElhSQCgQ1w1Bv/bfe1mfwlIIjDGLDTGXGptP2qM+dLazjLGXGOMaW6M6WyM0XXnlFJB1Sw2GoBWF8Q4RzGDI1nUr16J1Ef6+z2G+Rv2scHDgLeFm/b55fV05WellCrgx7/3YcbYPwGQYK11nncbau3oSG7q5lsTzWujkvJdVfjq42U7PO4fPWV5sc/lC7/1ESilVKhqWLOyc3vmHd05k51LVESYc9/jV7SjcmQ4ry78jRu7Neb9xec6ctMnDeV0dg6p6Yfo3rw2Lese56V5m4t8zdF/iufdX9ILLfPwkNbFr4wP9IpAKaUKEVZBqFQxzG1/3uynVyWdW779s7HdAIgMD6N7c8fUFk2tpqbCpE8ayoTL27Jh4iCvZVpdEMPtPZsWK3ZfaSJQSqnzcFF8TdInDSWxYXVm392Dt25MplNj9wFrrt6+Kdm5PbJzI7fjURGe/ySHVxA+v7N7yQIuhDYNKaVUCbWpX5U29at6PT6gTV36tKpDv9Z1+fHvfViwaR9/7tqYq5Ma5FshzdsMOw8MvDBf01Rpk1C7bT85OdmkpqYGOwyllPKLzXuPMeBfiwBY9nA/dh/OIqFBNbeBb8UlIiuMMcmejukVgVJKlSEt68bQq2UsPVrUpk5MFHViovz+mpoIlFKqjHnvls4BfT3tLFZKKZvTRKCUUjaniUAppWxOE4FSStmcJgKllLI5TQRKKWVzmgiUUsrmNBEopZTNaSJQSimb00SglFI2p4lAKaVsThOBUkrZnCYCpZSyOU0ESillc5oIlFLK5jQRKKWUzWkiUEopm9NEoJRSNue3RCAiUSKyTERWi8g6EXncQ5nGIvKdiKSJyEIRifNXPEoppTzz5xXBaaCvMaYDkAgMEpGuBcq8ALxvjEkAJgLP+DEepZRSHvgtERiH49bDCOvHFCjWBvjO2l4AXOGveJRSSnnm1z4CEQkTkVXAPmCeMWZpgSKrgaut7WFAjIjU8nCeMSKSKiKpmZmZ/gxZKaVsx6+JwBiTY4xJBOKAziLSrkCRB4BeIvIr0AvYBWR7OM8bxphkY0xybGysP0NWSinbCQ/EixhjDovIQmAQsNZl/27gKgARiQauNsYcCURMSimlHPx511CsiFS3tisB/YGNBcrUFpG8GMYD7/grHqWUUp75s2moHrBARNKA5Tj6CL4WkYkicrlVpjewSUQ2A3WBp/wYj1JKKQ/81jRkjEkDOnrY/6jL9gxghr9iUEopVTQdWayUUjaniUAppWxOE4FSStmcJgKllLI5TQRKKWVzmgiUUsrmNBEopZTNaSJQSimb00SglFI2p4lAKaVsThOBUkrZnCYCpZSyOU0ESillc5oIlFLK5jQRKKWUzWkiUEopm9NEoJRSNqeJQCmlbE4TgVJK2ZwmAqWUsjlNBEopZXOaCJRSyuY0ESillM1pIlBKKZvTRKCUUjaniUAppWzOb4lARKJEZJmIrBaRdSLyuIcyjURkgYj8KiJpIjLEX/EopZTyzJ9XBKeBvsaYDkAiMEhEuhYo8wgw3RjTERgB/NeP8SillPIg3F8nNsYY4Lj1MML6MQWLAVWt7WrAbn/Fo5RSyjO/9hGISJiIrAL2AfOMMUsLFJkAjBKRDGA28Dcv5xkjIqkikpqZmenPkJVSynb8mgiMMTnGmEQgDugsIu0KFBkJvGuMiQOGAB+IiFtMxpg3jDHJxpjk2NhYf4aslFK2E5C7howxh4GFwKACh24FpltlFgNRQO1AxKSUUsrBn3cNxYpIdWu7EtAf2Fig2A6gn1WmNY5EoG0/SikVQH7rLAbqAe+JSBiOhDPdGPO1iEwEUo0xXwL3A2+KyH04Oo5HW53MSimlAsSfdw2lAR097H/UZXs90N1fMSillCqajixWSimb00SglFI2p4lAKaVsThOBUkrZnCYCpZSyOU0ESillc5oIlFLK5jQRKKWUzWkiUEopm9NEoJRSNqeJQCmlbE4TgVJK2ZwmAqWUsjlNBEopZXOaCJRSyuY0ESillM1pIlBKKZvTRKCUUjYnobZEsIhkAtvP8+m1gf2lGE4o0Drbg93qbLf6Qsnr3NgYE+vpQMglgpIQkVRjTHKw4wgkrbM92K3Odqsv+LfO2jSklFI2p4lAKaVszm6J4I1gBxAEWmd7sFud7VZf8GOdbdVHoJRSyp3drgiUUkoVoIlAKaVszjaJQEQGicgmEdkqIinBjqckRCRdRNaIyCoRSbX21RSReSKyxfq3hrVfRORlq95pIpLkcp6brPJbROSmYNXHExF5R0T2ichal32lVkcR6WS9h1ut50pga+jOS50niMgu67NeJSJDXI6Nt+LfJCIDXfZ7/F0XkSYistR6Lz4RkYqBq51nItJQRBaIyAYRWSci91j7y+VnXUh9g/s5G2PK/Q8QBvwGNAUqAquBNsGOqwT1SQdqF9j3HJBibacAz1rbQ4A5gABdgaXW/prA79a/NaztGsGum0t9egJJwFp/1BFYBnSznjMHGFxG6zwBeMBD2TbW73Ek0MT6/Q4r7HcdmA6MsLZfA8aWgTrXA5Ks7Rhgs1W3cvlZF1LfoH7Odrki6AxsNcb8bow5A0wDrghyTKXtCuA9a/s94EqX/e8bhyVAdRGpBwwE5hljDhpjDgHzgEGBDtobY8wi4GCB3aVSR+tYVWPMYuP43/K+y7mCxkudvbkCmGaMOW2M2QZsxfF77vF33foW3BeYYT3f9f0LGmPMHmPMSmv7GLABaEA5/awLqa83Afmc7ZIIGgA7XR5nUPibX9YZ4FsRWSEiY6x9dY0xe8DxywbUsfZ7q3sovielVccG1nbB/WXVXVYzyDt5TSQUv861gMPGmOwC+8sMEYkHOgJLscFnXaC+EMTP2S6JwFObYCjfN9vdGJMEDAbuFJGehZT1Vvfy9J4Ut46hVPdXgWZAIrAHeNHaX67qLCLRwGfAvcaYo4UV9bAv5Ortob5B/ZztkggygIYuj+OA3UGKpcSMMbutf/cBM3FcJu61LoOx/t1nFfdW91B8T0qrjhnWdsH9ZY4xZq8xJscYkwu8ieOzhuLXeT+OZpTwAvuDTkQicPxRnGqM+Z+1u9x+1p7qG+zP2S6JYDnQwupNrwiMAL4MckznRUSqiEhM3jYwAFiLoz55d0rcBHxhbX8J3GjdbdEVOGJdas8FBohIDesydIC1rywrlTpax46JSFerTfVGl3OVKXl/DC3DcHzW4KjzCBGJFJEmQAscnaIef9et9vEFwHDr+a7vX9BY7//bwAZjzEsuh8rlZ+2tvkH/nIPVex7oHxx3G2zG0dP+cLDjKUE9muK4Q2A1sC6vLjjaBr8Dtlj/1rT2C/CKVe81QLLLuW7B0fm0Fbg52HUrUM+PcVwin8Xx7efW0qwjkGz9Z/sN+D+sUfZlsM4fWHVKs/4o1HMp/7AV/yZc7oTx9rtu/e4ss96LT4HIMlDni3E0XaQBq6yfIeX1sy6kvkH9nHWKCaWUsjm7NA0ppZTyQhOBUkrZnCYCpZSyOU0ESillc5oIlFLK5jQRqDJHRHKsGRhXi8hKEflTEeWri8gdPpx3oYjYasHzoojIuyIyvOiSqjzTRKDKolPGmERjTAdgPPBMEeWrA0UmgmBxGeWpVJmkiUCVdVWBQ+CYn0VEvrOuEtaISN4MspOAZtZVxPNW2b9bZVaLyCSX810jIstEZLOI9LDKhonI8yKy3Jr06y/W/noissg679q88q7EsTbEs9Y5l4lIc2v/uyLykogsAJ4Vx/z6n1vnXyIiCS51mmLFmiYiV1v7B4jIYquun1pz0yAik0RkvVX2BWvfNVZ8q0VkURF1EhH5P+scszg3mZuys2CPLNQf/Sn4A+TgGHG5ETgCdLL2h+OYUhigNo6RkwLEk38O/8HAL0Bl63HeqNSFwIvW9hBgvrU9BnjE2o4EUnHM/X4/50ZuhwExHmJNdylzI/C1tf0u8DUQZj3+D/CYtd0XWGVtPwtMdjlfDatui4Aq1r5/AI/imGt/E+fWGq9u/bsGaFBgn7c6XYVjiuYwoD5wGBge7M9cf4L7o5esqiw6ZYxJBBCRbsD7ItIOxx/9p8Ux22oujul163p4fn9gijHmJIAxxnWO/7xJzVbgSCDgmJcmwaWtvBqOOV2WA++IY5Kwz40xq7zE+7HLv/9y2f+pMSbH2r4YuNqK53sRqSUi1axYR+Q9wRhzSEQuxbEgyc+OqWmoCCwGjgJZwFvWt/mvraf9DLwrItNd6uetTj2Bj624dovI917qpGxEE4Eq04wxi0WkNhCL41t8LI4rhLMikg5EeXia4H3q3dPWvzmc+/0X4G/GGLdJ96ykMxT4QESeN8a87ylML9snCsTk6XmeYhUci6yM9BBPZ6AfjuRxF9DXGPNXEelixblKRBK91UkcSyDqvDIqH+0jUGWaiLTC0YxxAMe32n1WEugDNLaKHcOx7F+eb4FbRKSydY6aRbzMXGCs9c0fEWkpjlleG1uv9yaOGSOTvDz/Opd/F3spswi4wTp/b2C/ccxD/y2OP+h59a0BLAG6u/Q3VLZiigaqGWNmA/fimLseEWlmjFlqjHkUxzTEDb3VyYpjhNWHUA/oU8R7o2xArwhUWVRJRPKaYQS4yRiTIyJTga9EJJVzfQgYYw6IyM/iWPR9jjHmQetbcaqInAFmAw8V8npv4WgmWimOtphMHMv79QYeFJGzwHEcfQCeRIrIUhxfrNy+xVsmAFNEJA04ybkplp8EXrFizwEeN8b8T0RGAx+LSKRV7hEcCe8LEYmy3pf7rGPPi0gLa993OGamTfNSp5k4+ijW4Ji58odC3hdlEzr7qFIlYDVPJRtj9gc7FqXOlzYNKaWUzekVgVJK2ZxeESillM1pIlBKKZvTRKCUUjaniUAppWxOE4FSStnc/wMcx+/WYgIIHwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot_losses()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='progress-bar-interrupted' max='25030', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      Interrupted\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-1ceec9ac0fe5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_one_cycle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1e-3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmoms\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.8\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0.7\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'heb_spm_3'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecorder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot_losses\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/train.py\u001b[0m in \u001b[0;36mfit_one_cycle\u001b[0;34m(learn, cyc_len, max_lr, moms, div_factor, pct_start, final_div, wd, callbacks, tot_epochs, start_epoch)\u001b[0m\n\u001b[1;32m     21\u001b[0m     callbacks.append(OneCycleScheduler(learn, max_lr, moms=moms, div_factor=div_factor, pct_start=pct_start,\n\u001b[1;32m     22\u001b[0m                                        final_div=final_div, tot_epochs=tot_epochs, start_epoch=start_epoch))\n\u001b[0;32m---> 23\u001b[0;31m     \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcyc_len\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_lr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m def fit_fc(learn:Learner, tot_epochs:int=1, lr:float=defaults.lr,  moms:Tuple[float,float]=(0.95,0.85), start_pct:float=0.72,\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, epochs, lr, wd, callbacks)\u001b[0m\n\u001b[1;32m    198\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mwd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    199\u001b[0m         \u001b[0mcallbacks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallback_fns\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextra_callback_fns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 200\u001b[0;31m         \u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    201\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    202\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcreate_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m->\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(epochs, learn, callbacks, metrics)\u001b[0m\n\u001b[1;32m     99\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0myb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprogress_bar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_dl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpbar\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m                 \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 101\u001b[0;31m                 \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss_func\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    102\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mloss_batch\u001b[0;34m(model, xb, yb, loss_func, opt, cb_handler)\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mopt\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m         \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mskip_bwd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_backward_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mskip_bwd\u001b[0m\u001b[0;34m:\u001b[0m                     \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_backward_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mopt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.7/site-packages/fastai/callback.py\u001b[0m in \u001b[0;36mon_backward_begin\u001b[0;34m(self, loss)\u001b[0m\n\u001b[1;32m    288\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mon_backward_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m->\u001b[0m\u001b[0mTuple\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mAny\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mAny\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    289\u001b[0m         \u001b[0;34m\"Handle gradient calculation on `loss`.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 290\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msmoothener\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    291\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'last_loss'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'smooth_loss'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msmoothener\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msmooth\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    292\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'backward_begin'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcall_mets\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, 1e-3, moms=(0.8,0.7))\n",
    "learn.save('heb_spm_3')\n",
    "learn.recorder.plot_losses()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "the result seem very promising! <br>\n",
    "whithout touch the data at all, and with 3 epoch, we achive very Reasonable results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "במהלך השנה 1948 קמה מדינת ישראל על בסיס ב-1941 , ונמשכה עד 1939 . \n",
      "  בשנת 1996 , לאחר פירוק הציונות הדתית , הוקם לייצוא מנגנון צמצום כללי של עליית הנוער . \n",
      "  מחברת למלאכה לייצר רק צלילים של שאלה , וגייס ענייני אידאולוגיות לכולם .\n"
     ]
    }
   ],
   "source": [
    "TEXT = \"במהלך השנה 1948 קמה מדינת ישראל\"\n",
    "N_WORDS = 40\n",
    "N_SENTENCES = 1\n",
    "print(\"\\n\".join(learn.predict(TEXT, N_WORDS, temperature=0.9) for _ in range(N_SENTENCES)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## export model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# wiki_data = load_data(path, 'wiki_data.pkl')\n",
    "# learn = language_model_learner(wiki_data, AWD_LSTM)\n",
    "# learn.load(\"heb_model_save_4\")\n",
    "learn.export(\"wiki_heb_spm.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
